/* 
 * @Author: jade
 * @Date:   2015-12-21 10:33:08
 * @Last Modified by:   jade
 * @Last Modified time: 2016-01-13 11:23:42
 */
var Crawler = require("crawler");
var fs = require('fs');
var path = require('path');
var argv = require('yargs').argv;
var city = argv.city;
var Entities = require('html-entities').XmlEntities;
//node lianjia.js --city sh
//sh:上海，
var rootPath = 'http://' + city + '.lianjia.com';
var page = 1;

var cityMap = {
    bj: '北京',
    sh: '上海'
}

function store(str, fileName) {
    fs.appendFile(fileName || './xuequ_' + city + '.txt', str + '\n', function(err) {
        err && console.log(err)
    });
}


function getRandom() {
    return ~~(Math.random() * 10) + 0.1
}

if (page == 1) {
    var tilte = '学校全名,别名,城市,区域,办学性质,名额规定,升学方式,详细地址,升学方式,对口学校,直升学校,直升率,派位学校,对口小区&楼栋,划片更新时间,招生简章标题,招生简章内容,简章更新时间,排序'
    store(tilte);
}

function addQ(url, timer, cb, text) {
    (function(url, timer, cb, text) {
        setTimeout(function() {
            console.log(text);
            c.queue(url);
            cb && cb();
        }, timer)
    })(url, timer, cb, text)
}

function processStr(str) {
    return str.replace(/,/g, '，').replace(/\n/g,'');
}
var sort = 1;

var c = new Crawler({
    maxConnections: 10,
    //skipDuplicates: true,
    // This will be called for each crawled page
    callback: function(error, result, $) {
        var url = result.uri;

        if (url.indexOf(rootPath + '/xuequfang/pg')) {
            var str = [];
            //学校全名
            var $school = $('.school-cun');
            str.push($school.find('.name h1').text())
            str.push($school.find('.name').children().remove().end().text());
            str.push(cityMap[city]);
            str.push($($('.intro a')[2]).text().replace('学区房',''))
            str.push($($('.trait li')[0]).children().remove().end().text());
            str.push($($('.trait li')[1]).children().remove().end().text());
            str.push($($('.trait li')[2]).children().remove().end().text());
            str.push($($('.trait li')[3]).children().remove().end().text());

            str[8] = '';
            str[9] = '';
            str[10] = '';
            str[11] = '';
            str[12] = '';
            //升学相关 8
            $('.school li').each(function(itemIndex, item) {
                if ($(item).find('span').text() == '升学方式：') {
                    str[8] = $(item).find('label').text();
                }
                if ($(item).find('span').text() == '对口学校：') {
                    str[9] = $(item).find('label').text();
                }
                if ($(item).find('span').text() == '直升学校：') {
                    var $i = $(item).find('label i');
                    if ($i.length > 0) {
                        str[11] = $i.text();
                        str[10] = $(item).find('label').children().remove().end().text();
                    } else {
                        str[10] = $(item).find('label').text();
                    }
                }
                if ($(item).find('span').text() == '派位学校：') {
                    str[12] = $(item).find('label').text();
                }
            });

            var list = '';

            $('#list').parent().find('li').each(function(itemIndex, item) {
                list += $(item).find('.sp01').text() + '&&&&' + $(item).find('.sp02').text() + '||||';
            });

            str[13] = list;
            str[14] = $('#list span').text().replace(' 更新','');

            var $first = $('.sort .sort-box').first();
            str[17] = $first.find('.name span').text().replace(' 更新','');
            str[15] = $first.find('.name').children().remove().end().text();
            var entities = new Entities();

            var jz = $first.find('.cuntex').find('.see-comments').remove().end().html() || '';
            str[16] = processStr(entities.decode(jz));

            str.push(url.split('?')[1]);
            console.log(str)

            store(str.join(','));

        } else {
            //学区列表页面
            $('.house-lst .host-count h2 a').each(function(itemIndex, item) {
                var href = $(item).attr('href');
                var url = rootPath + href + '?' + ((page - 1) * 30 + itemIndex + 1);

                var timer = 2000 * (itemIndex + 1);
                addQ(url, timer, function() {}, '111111:' + timer / 1000 + 's后开始爬第' + ((page - 1) * 30 + itemIndex + 1) + '条学区房：')
            })
            page++;
            var timer = 1000 * 60;
            if (page < 10) {
                var nurl = rootPath + '/xuequfang/pg' + page + '/';
                addQ(nurl, timer, function() {}, '111111:' + timer / 1000 + 's后开始爬' + page + '页：')
                console.log('=================:' + page + '==============')
                console.log(nurl);
            }

        }
    }
});

c.queue(rootPath + '/xuequfang/pg1/')

//c.queue('http://www.iwjw.com/sale/shanghai/');
/*setInterval(function(){
    c.queue('http://www.iwjw.com/seeHouseList/')
}, 5*1000)*/
